{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将kilometer当做类别变量处理试试,异常值用groupby处理,'匿名特征可以进一步处理一下'\n",
    "## 基础工具\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.special import jn\n",
    "from IPython.display import display, clear_output\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import itertools\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline\n",
    "\n",
    "## 模型预测的\n",
    "from sklearn import linear_model\n",
    "from sklearn import preprocessing\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor\n",
    "\n",
    "## 数据降维处理的\n",
    "from sklearn.decomposition import PCA,FastICA,FactorAnalysis,SparsePCA\n",
    "\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "\n",
    "## 参数搜索和评价的\n",
    "from sklearn.model_selection import GridSearchCV,cross_val_score,StratifiedKFold,train_test_split\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "import scipy.signal as signal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#处理异常值\n",
    "def smooth_cols(group,out_value,kind):\n",
    "    cols = ['power']\n",
    "    if kind == 'g':\n",
    "        for col in cols:\n",
    "            yes_no = (group[col]<out_value).astype('int')\n",
    "            new = yes_no * group[col]\n",
    "            group[col] = new.replace(0,group[col].quantile(q=0.995))\n",
    "        return group\n",
    "    if kind == 'l':\n",
    "        for col in cols:\n",
    "            yes_no = (group[col]>out_value).astype('int')\n",
    "            new = yes_no * group[col]\n",
    "            group[col] = new.replace(0,group[col].quantile(q=0.07))\n",
    "        return group        \n",
    "def date_proc(x):\n",
    "    m = int(x[4:6])\n",
    "    if m == 0:\n",
    "        m = 1\n",
    "    return x[:4] + '-' + str(m) + '-' + x[6:]\n",
    "\n",
    "#定义日期提取函数\n",
    "def date_tran(df,fea_col):\n",
    "    for f in tqdm(fea_col):\n",
    "        df[f] = pd.to_datetime(df[f].astype('str').apply(date_proc))\n",
    "        df[f + '_year'] = df[f].dt.year\n",
    "        df[f + '_month'] = df[f].dt.month\n",
    "        df[f + '_day'] = df[f].dt.day\n",
    "        df[f + '_dayofweek'] = df[f].dt.dayofweek\n",
    "    return (df)\n",
    "\n",
    "#分桶操作\n",
    "def cut_group(df,cols,num_bins=50):\n",
    "    for col in cols:\n",
    "        all_range = int(df[col].max()-df[col].min())\n",
    "        bin = [i*all_range/num_bins for i in range(all_range)]\n",
    "        df[col+'_bin'] = pd.cut(df[col], bin, labels=False)\n",
    "    return df\n",
    "\n",
    "### count编码\n",
    "def count_coding(df,fea_col):\n",
    "    for f in fea_col:\n",
    "        df[f + '_count'] = df[f].map(df[f].value_counts())\n",
    "    return(df)\n",
    "#定义交叉特征统计\n",
    "def cross_cat_num(df,num_col,cat_col):\n",
    "    for f1 in tqdm(cat_col):\n",
    "        g = df.groupby(f1, as_index=False)\n",
    "        for f2 in tqdm(num_col):\n",
    "            feat = g[f2].agg({\n",
    "                '{}_{}_max'.format(f1, f2): 'max', '{}_{}_min'.format(f1, f2): 'min',\n",
    "                '{}_{}_median'.format(f1, f2): 'median',\n",
    "            })\n",
    "            df = df.merge(feat, on=f1, how='left')\n",
    "    return(df)\n",
    "### 类别特征的二阶交叉\n",
    "from scipy.stats import entropy\n",
    "def cross_qua_cat_num(df):\n",
    "    for f_pair in tqdm([\n",
    "        ['model', 'brand'], ['model', 'regionCode'], ['brand', 'regionCode']\n",
    "    ]):\n",
    "        ### 共现次数\n",
    "        df['_'.join(f_pair) + '_count'] = df.groupby(f_pair)['SaleID'].transform('count')\n",
    "        ### n unique、熵\n",
    "        df = df.merge(df.groupby(f_pair[0], as_index=False)[f_pair[1]].agg({\n",
    "            '{}_{}_nunique'.format(f_pair[0], f_pair[1]): 'nunique',\n",
    "            '{}_{}_ent'.format(f_pair[0], f_pair[1]): lambda x: entropy(x.value_counts() / x.shape[0])\n",
    "        }), on=f_pair[0], how='left')\n",
    "        df = df.merge(df.groupby(f_pair[1], as_index=False)[f_pair[0]].agg({\n",
    "            '{}_{}_nunique'.format(f_pair[1], f_pair[0]): 'nunique',\n",
    "            '{}_{}_ent'.format(f_pair[1], f_pair[0]): lambda x: entropy(x.value_counts() / x.shape[0])\n",
    "        }), on=f_pair[1], how='left')\n",
    "        ### 比例偏好\n",
    "        df['{}_in_{}_prop'.format(f_pair[0], f_pair[1])] = df['_'.join(f_pair) + '_count'] / df[f_pair[1] + '_count']\n",
    "        df['{}_in_{}_prop'.format(f_pair[1], f_pair[0])] = df['_'.join(f_pair) + '_count'] / df[f_pair[0] + '_count']\n",
    "    return (df)\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() \n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() \n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 37200128.00 MB\n",
      "Memory usage after optimization is: 10200232.00 MB\n",
      "Decreased by 72.6%\n",
      "Memory usage of dataframe is 12000128.00 MB\n",
      "Memory usage after optimization is: 3200232.00 MB\n",
      "Decreased by 73.3%\n",
      "Train data shape: (150000, 31)\n",
      "TestA data shape: (50000, 30)\n",
      "concat_data shape: (200000, 31)\n"
     ]
    }
   ],
   "source": [
    "## 通过Pandas对于数据进行读取 (pandas是一个很友好的数据读取函数库)\n",
    "Train_data = reduce_mem_usage(pd.read_csv('DataSet/used_car_train_20200313.csv', sep=' '))\n",
    "TestA_data = reduce_mem_usage(pd.read_csv('DataSet/used_car_testB_20200421.csv', sep=' '))\n",
    "\n",
    "#Train_data = Train_data[Train_data['price']>100]\n",
    "#Train_data['price'] = np.log1p(Train_data['price'])\n",
    "## 输出数据的大小信息\n",
    "print('Train data shape:',Train_data.shape)\n",
    "print('TestA data shape:',TestA_data.shape)\n",
    "\n",
    "\n",
    "#合并数据集\n",
    "concat_data = pd.concat([Train_data,TestA_data])\n",
    "concat_data['notRepairedDamage'] = concat_data['notRepairedDamage'].replace('-',0).astype('float16')\n",
    "concat_data = concat_data.fillna(concat_data.mode().iloc[0,:])\n",
    "#concat_data.index = range(200000)\n",
    "#concat_data = concat_data.groupby('bodyType').apply(smooth_cols,out_value=600,kind='g')\n",
    "#concat_data.index = range(200000)\n",
    "#concat_data['power'] = np.log(concat_data['power'])\n",
    "print('concat_data shape:',concat_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#截断异常值\n",
    "concat_data['power'][concat_data['power']>600] = 600\n",
    "concat_data['power'][concat_data['power']<1] = 1\n",
    "\n",
    "concat_data['v_13'][concat_data['v_13']>6] = 6\n",
    "concat_data['v_14'][concat_data['v_14']>4] = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 353)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in ['v_' +str(i) for i in range(14)]:\n",
    "    for j in ['v_' +str(i) for i in range(14)]:\n",
    "        concat_data[str(i)+'+'+str(j)] = concat_data[str(i)]+concat_data[str(j)]\n",
    "for i in ['model','brand', 'bodyType', 'fuelType','gearbox', 'power', 'kilometer', 'notRepairedDamage', 'regionCode']:\n",
    "    for j in ['v_' +str(i) for i in range(14)]:\n",
    "        concat_data[str(i)+'*'+str(j)] = concat_data[i]*concat_data[j]    \n",
    "concat_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\n# 对类别较少的特征采用one-hot编码\\none_hot_list = ['fuelType','gearbox','notRepairedDamage','bodyType','creatDate_year',]\\nfor col in one_hot_list:\\n    one_hot = pd.get_dummies(concat_data[col])\\n    one_hot.columns = [col+'_'+str(i) for i in range(len(one_hot.columns))]\\n    concat_data = pd.concat([concat_data,one_hot],axis=1)\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#提取日期信息\n",
    "date_cols = ['regDate', 'creatDate']\n",
    "concat_data = date_tran(concat_data,date_cols)\n",
    "\n",
    "'''\n",
    "# 对类别较少的特征采用one-hot编码\n",
    "one_hot_list = ['fuelType','gearbox','notRepairedDamage','bodyType','creatDate_year',]\n",
    "for col in one_hot_list:\n",
    "    one_hot = pd.get_dummies(concat_data[col])\n",
    "    one_hot.columns = [col+'_'+str(i) for i in range(len(one_hot.columns))]\n",
    "    concat_data = pd.concat([concat_data,one_hot],axis=1)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = concat_data.copy()\n",
    "\n",
    "#count编码\n",
    "count_list = ['regDate', 'creatDate', 'model', 'brand', 'regionCode','bodyType','fuelType','name','regDate_year', 'regDate_month', 'regDate_day',\n",
    "       'regDate_dayofweek' , 'creatDate_month','creatDate_day', 'creatDate_dayofweek','kilometer']\n",
    "       \n",
    "data = count_coding(data,count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#特征构造\n",
    "# 使用时间：data['creatDate'] - data['regDate']，反应汽车使用时间，一般来说价格与使用时间成反比\n",
    "# 不过要注意，数据里有时间出错的格式，所以我们需要 errors='coerce'\n",
    "data['used_time1'] = (pd.to_datetime(data['creatDate'], format='%Y%m%d', errors='coerce') - \n",
    "                            pd.to_datetime(data['regDate'], format='%Y%m%d', errors='coerce')).dt.days\n",
    "data['used_time2'] = (pd.datetime.now() - pd.to_datetime(data['regDate'], format='%Y%m%d', errors='coerce')).dt.days                        \n",
    "data['used_time3'] = (pd.datetime.now() - pd.to_datetime(data['creatDate'], format='%Y%m%d', errors='coerce') ).dt.days\n",
    "\n",
    "#分桶操作\n",
    "cut_cols = ['power']+['used_time1','used_time2','used_time3']\n",
    "data = cut_group(data,cut_cols,50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                            | 0/3 [00:00<?, ?it/s]\n",
      "  0%|                                                                                            | 0/6 [00:00<?, ?it/s]\u001b[A\n",
      " 17%|██████████████                                                                      | 1/6 [00:02<00:14,  2.82s/it]\u001b[A\n",
      " 33%|████████████████████████████                                                        | 2/6 [00:03<00:08,  2.18s/it]\u001b[A\n",
      " 50%|██████████████████████████████████████████                                          | 3/6 [00:04<00:05,  1.74s/it]\u001b[A\n",
      " 67%|████████████████████████████████████████████████████████                            | 4/6 [00:04<00:02,  1.43s/it]\u001b[A\n",
      " 83%|██████████████████████████████████████████████████████████████████████              | 5/6 [00:05<00:01,  1.21s/it]\u001b[A\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:06<00:00,  1.05s/it]\u001b[A\n",
      " 33%|████████████████████████████                                                        | 1/3 [00:06<00:12,  6.43s/it]\n",
      "  0%|                                                                                            | 0/6 [00:00<?, ?it/s]\u001b[A\n",
      " 17%|██████████████                                                                      | 1/6 [00:03<00:16,  3.22s/it]\u001b[A\n",
      " 33%|████████████████████████████                                                        | 2/6 [00:03<00:09,  2.45s/it]\u001b[A\n",
      " 50%|██████████████████████████████████████████                                          | 3/6 [00:04<00:05,  1.91s/it]\u001b[A\n",
      " 67%|████████████████████████████████████████████████████████                            | 4/6 [00:05<00:03,  1.54s/it]\u001b[A\n",
      " 83%|██████████████████████████████████████████████████████████████████████              | 5/6 [00:05<00:01,  1.28s/it]\u001b[A\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:06<00:00,  1.10s/it]\u001b[A\n",
      " 67%|████████████████████████████████████████████████████████                            | 2/3 [00:13<00:06,  6.58s/it]\n",
      "  0%|                                                                                            | 0/6 [00:00<?, ?it/s]\u001b[A\n",
      " 17%|██████████████                                                                      | 1/6 [00:03<00:19,  3.81s/it]\u001b[A\n",
      " 33%|████████████████████████████                                                        | 2/6 [00:04<00:11,  2.88s/it]\u001b[A\n",
      " 50%|██████████████████████████████████████████                                          | 3/6 [00:05<00:06,  2.22s/it]\u001b[A\n",
      " 67%|████████████████████████████████████████████████████████                            | 4/6 [00:05<00:03,  1.76s/it]\u001b[A\n",
      " 83%|██████████████████████████████████████████████████████████████████████              | 5/6 [00:06<00:01,  1.45s/it]\u001b[A\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:07<00:00,  1.22s/it]\u001b[A\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:21<00:00,  7.03s/it]\n"
     ]
    }
   ],
   "source": [
    "### 用数值特征对类别特征做统计刻画，随便挑了几个跟price相关性最高的匿名特征\n",
    "cross_cat = ['model', 'brand','regDate_year']\n",
    "cross_num = ['v_0','v_3', 'v_4', 'v_8', 'v_12','power']\n",
    "data = cross_cat_num(data,cross_num,cross_cat)#一阶交叉\n",
    "#data = cross_qua_cat_num(data)#二阶交叉"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 选择特征列\n",
    "numerical_cols = data.columns\n",
    "#print(numerical_cols)\n",
    "\n",
    "cat_fea = ['SaleID','offerType','seller']\n",
    "feature_cols = [col for col in numerical_cols if col not in cat_fea]\n",
    "feature_cols = [col for col in feature_cols if col not in ['price']]\n",
    "\n",
    "## 提前特征列，标签列构造训练样本和测试样本\n",
    "X_data = data.iloc[:len(Train_data),:][feature_cols]\n",
    "Y_data = Train_data['price']\n",
    "X_test  = data.iloc[len(Train_data):,:][feature_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import StratifiedKFold,KFold\n",
    "from itertools import product\n",
    "class MeanEncoder:\n",
    "    def __init__(self, categorical_features, n_splits=10, target_type='classification', prior_weight_func=None):\n",
    "        \"\"\"\n",
    "        :param categorical_features: list of str, the name of the categorical columns to encode\n",
    " \n",
    "        :param n_splits: the number of splits used in mean encoding\n",
    " \n",
    "        :param target_type: str, 'regression' or 'classification'\n",
    " \n",
    "        :param prior_weight_func:\n",
    "        a function that takes in the number of observations, and outputs prior weight\n",
    "        when a dict is passed, the default exponential decay function will be used:\n",
    "        k: the number of observations needed for the posterior to be weighted equally as the prior\n",
    "        f: larger f --> smaller slope\n",
    "        \"\"\"\n",
    " \n",
    "        self.categorical_features = categorical_features\n",
    "        self.n_splits = n_splits\n",
    "        self.learned_stats = {}\n",
    " \n",
    "        if target_type == 'classification':\n",
    "            self.target_type = target_type\n",
    "            self.target_values = []\n",
    "        else:\n",
    "            self.target_type = 'regression'\n",
    "            self.target_values = None\n",
    " \n",
    "        if isinstance(prior_weight_func, dict):\n",
    "            self.prior_weight_func = eval('lambda x: 1 / (1 + np.exp((x - k) / f))', dict(prior_weight_func, np=np))\n",
    "        elif callable(prior_weight_func):\n",
    "            self.prior_weight_func = prior_weight_func\n",
    "        else:\n",
    "            self.prior_weight_func = lambda x: 1 / (1 + np.exp((x - 2) / 1))\n",
    " \n",
    "    @staticmethod\n",
    "    def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_weight_func):\n",
    "        X_train = X_train[[variable]].copy()\n",
    "        X_test = X_test[[variable]].copy()\n",
    " \n",
    "        if target is not None:\n",
    "            nf_name = '{}_pred_{}'.format(variable, target)\n",
    "            X_train['pred_temp'] = (y_train == target).astype(int)  # classification\n",
    "        else:\n",
    "            nf_name = '{}_pred'.format(variable)\n",
    "            X_train['pred_temp'] = y_train  # regression\n",
    "        prior = X_train['pred_temp'].mean()\n",
    " \n",
    "        col_avg_y = X_train.groupby(by=variable, axis=0)['pred_temp'].agg([('mean', 'mean'), ('beta', 'size')])\n",
    "        col_avg_y['beta'] = prior_weight_func(col_avg_y['beta'])\n",
    "        col_avg_y[nf_name] = col_avg_y['beta'] * prior + (1 - col_avg_y['beta']) * col_avg_y['mean']\n",
    "        col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True)\n",
    " \n",
    "        nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values\n",
    "        nf_test = X_test.join(col_avg_y, on=variable).fillna(prior, inplace=False)[nf_name].values\n",
    " \n",
    "        return nf_train, nf_test, prior, col_avg_y\n",
    " \n",
    "    def fit_transform(self, X, y):\n",
    "        \"\"\"\n",
    "        :param X: pandas DataFrame, n_samples * n_features\n",
    "        :param y: pandas Series or numpy array, n_samples\n",
    "        :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n",
    "        \"\"\"\n",
    "        X_new = X.copy()\n",
    "        if self.target_type == 'classification':\n",
    "            skf = StratifiedKFold(self.n_splits)\n",
    "        else:\n",
    "            skf = KFold(self.n_splits)\n",
    " \n",
    "        if self.target_type == 'classification':\n",
    "            self.target_values = sorted(set(y))\n",
    "            self.learned_stats = {'{}_pred_{}'.format(variable, target): [] for variable, target in\n",
    "                                  product(self.categorical_features, self.target_values)}\n",
    "            for variable, target in product(self.categorical_features, self.target_values):\n",
    "                nf_name = '{}_pred_{}'.format(variable, target)\n",
    "                X_new.loc[:, nf_name] = np.nan\n",
    "                for large_ind, small_ind in skf.split(y, y):\n",
    "                    nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n",
    "                        X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, target, self.prior_weight_func)\n",
    "                    X_new.iloc[small_ind, -1] = nf_small\n",
    "                    self.learned_stats[nf_name].append((prior, col_avg_y))\n",
    "        else:\n",
    "            self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features}\n",
    "            for variable in self.categorical_features:\n",
    "                nf_name = '{}_pred'.format(variable)\n",
    "                X_new.loc[:, nf_name] = np.nan\n",
    "                for large_ind, small_ind in skf.split(y, y):\n",
    "                    nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n",
    "                        X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, None, self.prior_weight_func)\n",
    "                    X_new.iloc[small_ind, -1] = nf_small\n",
    "                    self.learned_stats[nf_name].append((prior, col_avg_y))\n",
    "        return X_new\n",
    " \n",
    "    def transform(self, X):\n",
    "        \"\"\"\n",
    "        :param X: pandas DataFrame, n_samples * n_features\n",
    "        :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n",
    "        \"\"\"\n",
    "        X_new = X.copy()\n",
    " \n",
    "        if self.target_type == 'classification':\n",
    "            for variable, target in product(self.categorical_features, self.target_values):\n",
    "                nf_name = '{}_pred_{}'.format(variable, target)\n",
    "                X_new[nf_name] = 0\n",
    "                for prior, col_avg_y in self.learned_stats[nf_name]:\n",
    "                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[\n",
    "                        nf_name]\n",
    "                X_new[nf_name] /= self.n_splits\n",
    "        else:\n",
    "            for variable in self.categorical_features:\n",
    "                nf_name = '{}_pred'.format(variable)\n",
    "                X_new[nf_name] = 0\n",
    "                for prior, col_avg_y in self.learned_stats[nf_name]:\n",
    "                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[\n",
    "                        nf_name]\n",
    "                X_new[nf_name] /= self.n_splits\n",
    " \n",
    "        return X_new\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_list = ['model','brand','name','regionCode']+date_cols\n",
    "MeanEnocodeFeature = class_list#声明需要平均数编码的特征\n",
    "ME = MeanEncoder(MeanEnocodeFeature,target_type='regression') #声明平均数编码的类\n",
    "X_data = ME.fit_transform(X_data,Y_data)#对训练数据集的X和y进行拟合\n",
    "#x_train_fav = ME.fit_transform(x_train,y_train_fav)#对训练数据集的X和y进行拟合\n",
    "X_test = ME.transform(X_test)#对测试集进行编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_data['price'] = Train_data['price']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:47<00:00,  7.98s/it]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "### target encoding目标编码，回归场景相对来说做目标编码的选择更多，不仅可以做均值编码，还可以做标准差编码、中位数编码等\n",
    "enc_cols = []\n",
    "stats_default_dict = {\n",
    "    'max': X_data['price'].max(),\n",
    "    'min': X_data['price'].min(),\n",
    "    'median': X_data['price'].median(),\n",
    "    'mean': X_data['price'].mean(),\n",
    "    'sum': X_data['price'].sum(),\n",
    "    'std': X_data['price'].std(),\n",
    "    'skew': X_data['price'].skew(),\n",
    "    'kurt': X_data['price'].kurt(),\n",
    "    'mad': X_data['price'].mad()\n",
    "}\n",
    "### 暂且选择这三种编码\n",
    "enc_stats = ['max','min','mean']\n",
    "skf = KFold(n_splits=10, shuffle=True, random_state=42)\n",
    "for f in tqdm(['regionCode','brand','regDate_year','creatDate_year','kilometer','model']):\n",
    "    enc_dict = {}\n",
    "    for stat in enc_stats:\n",
    "        enc_dict['{}_target_{}'.format(f, stat)] = stat\n",
    "        X_data['{}_target_{}'.format(f, stat)] = 0\n",
    "        X_test['{}_target_{}'.format(f, stat)] = 0\n",
    "        enc_cols.append('{}_target_{}'.format(f, stat))\n",
    "    for i, (trn_idx, val_idx) in enumerate(skf.split(X_data, Y_data)):\n",
    "        trn_x, val_x = X_data.iloc[trn_idx].reset_index(drop=True), X_data.iloc[val_idx].reset_index(drop=True)\n",
    "        enc_df = trn_x.groupby(f, as_index=False)['price'].agg(enc_dict)\n",
    "        val_x = val_x[[f]].merge(enc_df, on=f, how='left')\n",
    "        test_x = X_test[[f]].merge(enc_df, on=f, how='left')\n",
    "        for stat in enc_stats:\n",
    "            val_x['{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].fillna(stats_default_dict[stat])\n",
    "            test_x['{}_target_{}'.format(f, stat)] = test_x['{}_target_{}'.format(f, stat)].fillna(stats_default_dict[stat])\n",
    "            X_data.loc[val_idx, '{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].values \n",
    "            X_test['{}_target_{}'.format(f, stat)] += test_x['{}_target_{}'.format(f, stat)].values / skf.n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 454)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "drop_list = ['regDate', 'creatDate','brand_power_min', 'regDate_year_power_min']\n",
    "x_train = X_data.drop(drop_list+['price'],axis=1)\n",
    "x_test = X_test.drop(drop_list,axis=1)\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "#特征归一化\n",
    "min_max_scaler = MinMaxScaler()\n",
    "min_max_scaler.fit(pd.concat([x_train,x_test]).values)\n",
    "all_data = min_max_scaler.transform(pd.concat([x_train,x_test]).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import decomposition\n",
    "pca = decomposition.PCA(n_components=146)\n",
    "all_pca = pca.fit_transform(all_data)\n",
    "X_pca = all_pca[:len(x_train)]\n",
    "test = all_pca[len(x_train):]\n",
    "y = Train_data['price'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Conv1D, Activation, MaxPool1D, Flatten, Dense\n",
    "from keras.layers import Input, Dense, Concatenate, Reshape, Dropout, merge, Add\n",
    "def NN_model(input_dim):\n",
    "    init = keras.initializers.glorot_uniform(seed=1)\n",
    "    model = keras.models.Sequential()\n",
    "    model.add(Dense(units=300, input_dim=input_dim, kernel_initializer=init, activation='softplus'))\n",
    "    #model.add(Dropout(0.2))\n",
    "    model.add(Dense(units=300, kernel_initializer=init, activation='softplus'))\n",
    "    #model.add(Dropout(0.2))\n",
    "    model.add(Dense(units=64, kernel_initializer=init, activation='softplus'))\n",
    "    model.add(Dense(units=32, kernel_initializer=init, activation='softplus'))\n",
    "    model.add(Dense(units=8, kernel_initializer=init, activation='softplus'))\n",
    "    model.add(Dense(units=1))\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import Callback, EarlyStopping\n",
    "class Metric(Callback):\n",
    "    def __init__(self, model, callbacks, data):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.callbacks = callbacks\n",
    "        self.data = data\n",
    "\n",
    "    def on_train_begin(self, logs=None):\n",
    "        for callback in self.callbacks:\n",
    "            callback.on_train_begin(logs)\n",
    "\n",
    "    def on_train_end(self, logs=None):\n",
    "        for callback in self.callbacks:\n",
    "            callback.on_train_end(logs)\n",
    "\n",
    "    def on_epoch_end(self, batch, logs=None):\n",
    "        X_train, y_train = self.data[0][0], self.data[0][1]\n",
    "        y_pred3 = self.model.predict(X_train)\n",
    "        y_pred = np.zeros((len(y_pred3), ))\n",
    "        y_true = np.zeros((len(y_pred3), ))\n",
    "        for i in range(len(y_pred3)):\n",
    "            y_pred[i] = y_pred3[i]\n",
    "        for i in range(len(y_pred3)):\n",
    "            y_true[i] = y_train[i]\n",
    "        trn_s = mean_absolute_error(y_true, y_pred)\n",
    "        logs['trn_score'] = trn_s\n",
    "        \n",
    "        X_val, y_val = self.data[1][0], self.data[1][1]\n",
    "        y_pred3 = self.model.predict(X_val)\n",
    "        y_pred = np.zeros((len(y_pred3), ))\n",
    "        y_true = np.zeros((len(y_pred3), ))\n",
    "        for i in range(len(y_pred3)):\n",
    "            y_pred[i] = y_pred3[i]\n",
    "        for i in range(len(y_pred3)):\n",
    "            y_true[i] = y_val[i]\n",
    "        val_s = mean_absolute_error(y_true, y_pred)\n",
    "        logs['val_score'] = val_s\n",
    "        print('trn_score', trn_s, 'val_score', val_s)\n",
    "\n",
    "        for callback in self.callbacks:\n",
    "            callback.on_epoch_end(batch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "  \n",
    "def scheduler(epoch):\n",
    "    # 每隔100个epoch，学习率减小为原来的1/10\n",
    "    if epoch % 20 == 0 and epoch != 0:\n",
    "        lr = K.get_value(model.optimizer.lr)\n",
    "        K.set_value(model.optimizer.lr, lr * 0.6)\n",
    "        print(\"lr changed to {}\".format(lr * 0.6))\n",
    "    return K.get_value(model.optimizer.lr)\n",
    "reduce_lr = LearningRateScheduler(scheduler)\n",
    "#model.fit(train_x, train_y, batch_size=32, epochs=5, callbacks=[reduce_lr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold: 0\n",
      "Epoch 1/145\n",
      "63/63 - 2s - loss: 2485.2417 - mae: 2485.2417 - val_loss: 880.6960 - val_mae: 880.6960\n",
      "Epoch 2/145\n",
      "63/63 - 2s - loss: 782.8497 - mae: 782.8497 - val_loss: 793.3176 - val_mae: 793.3176\n",
      "Epoch 3/145\n",
      "63/63 - 2s - loss: 670.3107 - mae: 670.3107 - val_loss: 844.4489 - val_mae: 844.4489\n",
      "Epoch 4/145\n",
      "63/63 - 2s - loss: 664.0917 - mae: 664.0917 - val_loss: 571.1913 - val_mae: 571.1913\n",
      "Epoch 5/145\n",
      "63/63 - 2s - loss: 621.5558 - mae: 621.5558 - val_loss: 617.5803 - val_mae: 617.5803\n",
      "Epoch 6/145\n",
      "63/63 - 2s - loss: 606.2938 - mae: 606.2938 - val_loss: 678.8544 - val_mae: 678.8544\n",
      "Epoch 7/145\n",
      "63/63 - 2s - loss: 567.8410 - mae: 567.8410 - val_loss: 518.9509 - val_mae: 518.9509\n",
      "Epoch 8/145\n",
      "63/63 - 2s - loss: 515.2856 - mae: 515.2856 - val_loss: 506.4954 - val_mae: 506.4954\n",
      "Epoch 9/145\n",
      "63/63 - 2s - loss: 507.3657 - mae: 507.3657 - val_loss: 491.7051 - val_mae: 491.7051\n",
      "Epoch 10/145\n",
      "63/63 - 2s - loss: 513.0317 - mae: 513.0317 - val_loss: 509.8212 - val_mae: 509.8212\n",
      "Epoch 11/145\n",
      "63/63 - 2s - loss: 524.8220 - mae: 524.8220 - val_loss: 504.7672 - val_mae: 504.7672\n",
      "Epoch 12/145\n",
      "63/63 - 2s - loss: 530.0285 - mae: 530.0285 - val_loss: 625.7012 - val_mae: 625.7012\n",
      "Epoch 13/145\n",
      "63/63 - 2s - loss: 536.4235 - mae: 536.4235 - val_loss: 588.5021 - val_mae: 588.5021\n",
      "Epoch 14/145\n",
      "63/63 - 2s - loss: 546.4833 - mae: 546.4833 - val_loss: 503.9569 - val_mae: 503.9569\n",
      "Epoch 15/145\n",
      "63/63 - 2s - loss: 513.1696 - mae: 513.1696 - val_loss: 486.4309 - val_mae: 486.4309\n",
      "Epoch 16/145\n",
      "63/63 - 2s - loss: 492.6819 - mae: 492.6819 - val_loss: 488.8856 - val_mae: 488.8856\n",
      "Epoch 17/145\n",
      "63/63 - 2s - loss: 519.2598 - mae: 519.2598 - val_loss: 601.7971 - val_mae: 601.7971\n",
      "Epoch 18/145\n",
      "63/63 - 2s - loss: 534.3559 - mae: 534.3559 - val_loss: 590.4240 - val_mae: 590.4240\n",
      "Epoch 19/145\n",
      "63/63 - 2s - loss: 484.0637 - mae: 484.0637 - val_loss: 462.3629 - val_mae: 462.3629\n",
      "Epoch 20/145\n",
      "63/63 - 2s - loss: 467.7639 - mae: 467.7639 - val_loss: 469.2489 - val_mae: 469.2489\n",
      "lr changed to 0.008999999798834323\n",
      "Epoch 21/145\n",
      "63/63 - 2s - loss: 451.2892 - mae: 451.2892 - val_loss: 462.2690 - val_mae: 462.2690\n",
      "Epoch 22/145\n",
      "63/63 - 2s - loss: 454.6800 - mae: 454.6800 - val_loss: 442.6936 - val_mae: 442.6936\n",
      "Epoch 23/145\n",
      "63/63 - 2s - loss: 438.5912 - mae: 438.5912 - val_loss: 436.9290 - val_mae: 436.9290\n",
      "Epoch 24/145\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-23-3cd1c956c521>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     26\u001b[0m     model.fit(X_train, y_train, batch_size=b_size, epochs=max_epochs, \n\u001b[0;32m     27\u001b[0m               \u001b[0mvalidation_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 28\u001b[1;33m               callbacks=[reduce_lr], shuffle=True, verbose=2)\n\u001b[0m\u001b[0;32m     29\u001b[0m     \u001b[0my_pred3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_pred3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    106\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 108\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    109\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1096\u001b[0m                 batch_size=batch_size):\n\u001b[0;32m   1097\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1098\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1099\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1100\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    778\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 780\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    781\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    805\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    806\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 807\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    808\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2829\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2830\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2831\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1846\u001b[0m                            resource_variable_ops.BaseResourceVariable))],\n\u001b[0;32m   1847\u001b[0m         \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1848\u001b[1;33m         cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[0;32m   1849\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1850\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1922\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1923\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1924\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1926\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    548\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 550\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    551\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mD:\\software\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[1;32m---> 60\u001b[1;33m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "n_splits = 6\n",
    "kf = KFold(n_splits=n_splits, shuffle=True)\n",
    "\n",
    "import keras \n",
    "\n",
    "b_size = 2000\n",
    "max_epochs = 145\n",
    "oof_pred = np.zeros((len(X_pca), ))\n",
    "\n",
    "sub = pd.read_csv('used_car_testB_20200421.csv',sep = ' ')[['SaleID']].copy()\n",
    "sub['price'] = 0\n",
    "\n",
    "avg_mae = 0\n",
    "for fold, (trn_idx, val_idx) in enumerate(kf.split(X_pca, y)):\n",
    "    print('fold:', fold)\n",
    "    X_train, y_train = X_pca[trn_idx], y[trn_idx]\n",
    "    X_val, y_val = X_pca[val_idx], y[val_idx]\n",
    "    \n",
    "    model = NN_model(X_train.shape[1])\n",
    "    simple_adam = keras.optimizers.Adam(lr = 0.015)\n",
    "    \n",
    "    model.compile(loss='mae', optimizer=simple_adam,metrics=['mae'])\n",
    "    es = keras.callbacks.EarlyStopping(monitor='val_score', patience=10, verbose=2, mode='min', restore_best_weights=True)\n",
    "    es.set_model(model)\n",
    "    metric = Metric(model, [es], [(X_train, y_train), (X_val, y_val)])\n",
    "    model.fit(X_train, y_train, batch_size=b_size, epochs=max_epochs, \n",
    "              validation_data = (X_val, y_val),\n",
    "              callbacks=[reduce_lr], shuffle=True, verbose=2)\n",
    "    y_pred3 = model.predict(X_val)\n",
    "    y_pred = np.zeros((len(y_pred3), ))\n",
    "    sub['price'] += model.predict(test).reshape(-1,)/n_splits\n",
    "    for i in range(len(y_pred3)):\n",
    "        y_pred[i] = y_pred3[i]\n",
    "        \n",
    "    oof_pred[val_idx] = y_pred\n",
    "    val_mae = mean_absolute_error(y[val_idx], y_pred)\n",
    "    avg_mae += val_mae/n_splits\n",
    "    print()\n",
    "    print('val_mae is:{}'.format(val_mae))\n",
    "    print()\n",
    "mean_absolute_error(y, oof_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('Results/nn_sub_{}_{}.csv'.format('mae', sub['price'].mean()), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
